import main
import filtering_transport
import common
import output
import hierarchical

from collections import OrderedDict
import datetime
import numpy as np
import copy

#start_date = datetime.datetime(2014, 9, 9, 10, 0)
#end_date = start_date + datetime.timedelta(days = 5)
#start_date = start_date + datetime.timedelta(days = 1)

#start_date = datetime.datetime(2014, 9, 9, 0, 0)
#end_date = datetime.datetime(2014, 9, 11, 0, 0)

#start_date = datetime.datetime(2014, 9, 25, 0, 0)
#end_date = datetime.datetime(2014, 9, 27, 0, 0)

start_date = datetime.datetime(2014, 9, 10, 0, 0)
end_date = datetime.datetime(2014, 9, 11, 0, 0)

#start_date = datetime.datetime(2014, 10, 6, 0, 0)
#end_date = datetime.datetime(2014, 10, 8, 0, 0)

#start_date = datetime.datetime(2014, 10, 5, 0, 0)
#end_date = datetime.datetime(2014, 10, 7, 0, 0)

# READ DATA
[class_data, scan_data_binned] = main.load_data(start_date, end_date)
print len(scan_data_binned)
    
#output.plot_figure_scatter(scan_data_binned, class_data, start_date, end_date, "bin" + datetime.datetime.strftime(start_date, "%Y-%m-%d") + datetime.datetime.strftime(end_date, "%Y-%m-%d") + ".pdf", (8,12), True, {}, colours_list, True)

# FILTER TRANSPORT ROUTERS
min_sequence = 4
jaccard_threshold = 0.9
jaccard_delta = 1
min_time = 10

window_length = 15
min_window =  7

scan_data_binned_orig = copy.copy(scan_data_binned)

scan_data_binned = filtering_transport.remove_transport_routers_window(scan_data_binned, min_window, window_length, start_date, end_date)
print "Sequence window removal: ", len(scan_data_binned)=
output.plot_figure_scatter(scan_data_binned, class_data, start_date, end_date, "bin" + datetime.datetime.strftime(start_date, "%Y-%m-%d") + datetime.datetime.strftime(end_date, "%Y-%m-%d") + ".pdf", (8,4), True, {}, {})

#scan_data_binned = filtering_transport.remove_routers_by_jaccard(scan_data_binned, scan_data_binned_orig, jaccard_threshold, jaccard_delta)    
#print "Jaccard removal: ", len(scan_data_binned)  

vd = common.to_vector_data(scan_data_binned, start_date, end_date)

#k = 3
#ks, global_imp, fk, alfa = k_means_main.gap_statistic_my(vd)
#k = np.argmin(fk) + 1
#[mu, clustered_idx] = k_means_main.kcluster(vd, k)
#print clustered_idx

# HIERARCH

res = hierarchical.hierarchical_cluster(vd)
#where to cut hierarchical clustering
ks, global_imp, fk, alfa = hierarchical.gap_statistic_my(res)
k = np.argmin(fk)
[mu, res_optimal] = hierarchical.extract_clusters_with_mean_orig(res, ks[k])
colour_list = hierarchical.get_router_colors(scan_data_binned, res_optimal)

#output.plot_figure_1(scan_data_binned, jaccard_delta, class_data, start_date, end_date, "k_means_" + str(k) + "_jaccard_rejection_X" + datetime.datetime.strftime(start_date, "%Y-%m-%d") + datetime.datetime.strftime(end_date, "%Y-%m-%d") + ".pdf", (8,6), True, {}, colour_list)
#output.plot_figure_1(scan_data_binned, jaccard_delta, class_data, start_date, end_date, "hierarch_" + "_jaccard_rejection_X" + datetime.datetime.strftime(start_date, "%Y-%m-%d") + datetime.datetime.strftime(end_date, "%Y-%m-%d") + ".pdf", (8,6), True, {}, colour_list)
#
transportation_routers_prospect = OrderedDict()
for router in scan_data_binned_orig.keys():
    if router not in scan_data_binned.keys():
        transportation_routers_prospect[router] = scan_data_binned_orig[router]

#output.plot_figure_1(transportation_routers_prospect, jaccard_delta, class_data, start_date, end_date, "transportation_prospect_" + datetime.datetime.strftime(start_date, "%Y-%m-%d") + datetime.datetime.strftime(end_date, "%Y-%m-%d") + ".pdf", (8,6), True, {}, {})
#output.plot_figure_1(scan_data_binned, jaccard_delta, class_data, start_date, end_date, "rejection_" + datetime.datetime.strftime(start_date, "%Y-%m-%d") + datetime.datetime.strftime(end_date, "%Y-%m-%d") + ".pdf", (8,6), True, {})
location_vectors = hierarchical.get_location_vectors(scan_data_binned, res_optimal, start_date, end_date)    

to_remove = []
threshold = 0.2
tr_prospect = common.to_vector_data(transportation_routers_prospect, start_date, end_date)
for idx in xrange(len(tr_prospect)):
    if(common.close_to_location(tr_prospect[idx], location_vectors, threshold)):
        to_remove.append(transportation_routers_prospect.keys()[idx])
        
print len(transportation_routers_prospect)

for remove_router in to_remove:
    transportation_routers_prospect.pop(remove_router, None)

print len(transportation_routers_prospect)

#output.plot_figure_1(transportation_routers_prospect, jaccard_delta, class_data, start_date, end_date, "transportation_prospect_rejected_" + datetime.datetime.strftime(start_date, "%Y-%m-%d") + datetime.datetime.strftime(end_date, "%Y-%m-%d") + ".pdf", (8,6), True, {}, {})

#do directed networking on the transportation routers
#output.plot_router_network_directed(transportation_routers_prospect, class_data, "test.gexf")